var citiesList = "sh,bj";

var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});
//var cities = citiesList.split(",");

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/Communities';
var async = require('async');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');

//async.eachSeries(cities, function (city) {
cities.forEach(function(city){
    var url = 'http://' + city + '.sofang.com'
    var db = null;
    var dbCollection = 'xq_sf_' + city;
    var dbCollectionDetail = 'xq_sfDetail_' + city;

    var totalData = 0
    var dataCount = 0  // stores the existing count in the detail collection so that the crawler will skip over the existing data. Relies on Mongo collecting data in ascending order everytime
    var counter = 0


    function getExistingDataCount(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                assert.equal(null, err);
                //console.log("Connected correctly to server");
                var coll = db.collection(dbcollection);
                coll.count(function (err, count) {
                    totalData = count
                    console.log("Total data %s: %s", dbcollection, totalData)
                    var coll = db.collection(dbcollectionDetail);
                    coll.count(function (err, count) {
                        dataCount = count
                        console.log("Data already in %s: %s", dbcollectionDetail, dataCount)
                        callback();
                    });
                });
            });
    };

    function run() {
        var manifest = {
            target: {
                url: url,
                ranges: {
                    // number: [1, 1976, 1], // from 0 to 100 and step 5 numbers each time
                    // letter: ['a', 'z'] // from a to z, put {{letter}} where you want it
                    // resource: 'csv' // configure this resource and put {{resource:csv}}
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
//            resources: {
//                mongodb: {
//                    plugin: 'mongodb',
//                    options: {
////                collection: 'xq_shanghai',
//                        collection: dbCollection,
//                        field: 'commId'
//                    }
//                }
//            },
//            plugins: {
//                mongodb: {
//                    connection: connection
//                },
//                csv: true
//            },

            extractors: [{
                name: 'details',
                root: '//div[@class="property_c"]',
                //root:'',
                fields: {
                    //developedBy: '//dd[6]/text()',
                    //propertyType: '//dd[5]/text()[2]',
                    //managedBy: '//dd[8]/text()',
                    //fees: '//dd[7]/span[@class="ddinfo"]/text()',
                    //rentalPercentage: '//*[@id="peitao_3"]/ul/li[1]/span[3]',
                    //greenPercentage: '//*[@id="peitao_3"]/ul/li[2]/span[3]',
                    //constructionArea: '//*[@id="peitao_3"]/ul/li[4]/span[3]',
                    //area: '//*[@id="peitao_3"]/ul/li[3]/span[3]',
                    //dateBuilt: '//dd[5]/text()[1]',
                    //parkingUnits: '//*[@id="peitao_3"]/ul/li[5]/span[3]',
                }
            }]
        };

        var dbconn = null;

        function insertDocuments(doc, callback) {

            var collection = dbconn.collection(dbCollectionDetail)

            collection.insert(
                [doc]
                , function (err, result) {
                    //assert.equal(err, null);
                    //assert.equal(1, result.result.n);
                    //assert.equal(1, result.ops.length);
                    console.log("Inserted %s into collection %s", doc.name, dbCollectionDetail);
                    //callback(result);
                });
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
//            data.data.forEach(
//                function (doc) {
////                   insertDocuments(doc, function(){});
//                })
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            console.log('The job is done');
            db.close()
            // not exactly,
        });

        function findCommids(db, callback) {
            var collection = db.collection(dbCollection);
            collection.find({}).toArray(
                function (err, docs) {

//            console.log(docs);
                    callback(docs);
                });
        };

        var ids = [];

        function init(callback) {
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    //collection = db.collection(dbCollectionDetail)
                    callback(null, 'init');
                });
        }

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    ids = docs;
                    //docs.forEach(function(doc, i){
                    //    if(city == 'sh' && i > 23913){
                    //        ids.push(doc)
                    //    }
                    //})
                    //console.log(ids.length)
                    callback(null, 'getIds');
                }
            );
        }

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(ids, function (item, callbackInner) {
                    console.log("(%s: %s of %s)",city,counter,totalData)
                    counter++
                    if (counter - 1 < dataCount) {
                        callbackInner()
                        return
                    }

                    manifest.target.url = url + item.href;

                    //console.log(manifest.target.url)

                    var job1 = new Crawler(manifest);
                    //console.log(job1)

                    job1.on('data', function (data) {
                        var $ = cheerio.load(data.request.body);

                        var houseData = $('div.property_c')

                        var doc = {}

                        try{
                            var titleData = houseData.find('div.title')[0]
                            var map = titleData.children[1].children[3].children[1].attribs.href
                            var regex = /^(.*)longitude=(.*)&latitude=(.*)&communitypinyin=(.*)&communityname=(.*)(&?)/;
                            var result = map.match(regex);
                            doc.lon = parseFloat(result[2].replace(/[^0-9\.]+/g, ''))
                            doc.lat = parseFloat(result[3].replace(/[^0-9\.]+/g, ''))
                            doc.pinyin = result[4]

                            var otherData = houseData.find('div.house_msg')[0]
                            try{
                                doc.fees = otherData.children[3].children[3].children[1].data
                            }catch(e1){
                                console.log(e1)
                                doc.fees = 0
                            }
                            doc.propertyType = otherData.children[1].children[3].children[1].data
                            doc.manager = otherData.children[5].children[2].data.replace("\r\n\t\t","")
                            doc.developer = otherData.children[7].children[2].data.replace("\r\n\t\t","")

                            var priceData = houseData.find('div#roomPrice')[0]
                            var pdLen = priceData.children.length
                            for (var i =0 ; i < pdLen; i++){
                                doc['RoomPrice' + (i+1)]= parseInt(priceData.children[i].children[2].children[0].children[0].data)
                            }

                        }catch(e){
                            console.log(e)
                        }

                        // copying all fields from the non-detail docs ensures _id field replication, which prevents duplicates upon multiple reinserts and enables easy cross-checking between the detail and non-detail collections
                        for(var field in item){
                            doc[field] = item[field]
                            if (field == 'name')
                                doc[field] = doc[field].replace("小区","")
                            else if (field == 'commId')
                                doc[field] = parseInt(doc[field].replace(/[^0-9\.]+/g, ''))
                        }
                        doc.dateCreated = new Date()

                        insertDocuments(doc, function () {
                        })

                    });

                    job1.on('error', function (err) {
                        throw err;
                    });

                    job1.on('end', function () {

                        console.log('The job is done: %d', ++cnt);
                        delete job1;

                        callbackInner();
                    });

                    job1.start();

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);

            });
    }

    getExistingDataCount(run);
});

/*
 node /workspace/CrawlerJS/examples/cities/sofang-master-detail.js

 aq ay ab anshun al ak aks altd aomen alsm as
 sh sjz suzhou sm sr smx sq shiyan suizhou shaoyang shuozhou sg sz st sw sanya suining simao shannandi sl szs shz sy sp songyuan suihua su suqian sx
 kaifeng km klmy kzls ksd kunshan
 jdz jj ja jn jining jiaozuo jingmen jingzhou jincheng jz jm jy jyg jinchang jq jiangyin jinzhou jl jixi jms jx jh
 la lf ly lw linyi lc luoyang luohe ld liuzhou linfen lb luzhou lvliang ls liangshan lps lj lincang lasa linzhi lz ln lx liaoyang liaoyuan lyg lishui
 tj ts tongling ty ta tr tongliao tc tianshui tlfd tcd tw taicang tl th taizhou tz
 wuhu wf weihai wh wuzhou wuhai ws wn wuwei wuzhong wlmq wlcb wlcbm wx wz wenchang
 xt xuancheng xm xinyu xx xc xy xiaogan xianning xiangtan xiangxi xinzhou xa xianyang xn xam xg xiangyang xl xsbn xz
 yingtan yichun yt yq yc yy yiyang yongzhou yuncheng yj yf yulin yb ya yx yanan yl ys yinchuan yili yiwu yk yanbian yashan yich yancheng yz
 zjk zhangzhou zb zaozhuang zz zk zmd zhuzhou zjj zh zhanjiang za zs zg zy zunyi zt zhangye zw zjw zj zhoushan
 bd bb bozhou bz bh baise bazhong bt bj bijie bs baoji by bycem betl bygl bazhou bx baishan bc
 chengde chuzhou cangzhou ch chizhou changzhi cs changde chenzhou chaozhou chongzuo cq cd cxyz chifeng changdu cjhz changshu cy cc cz
 dy dz dt dg deyang dazhou dlbz dh diqing dx dl dd dq dxal ez estjzmz erds
 fy fz fuzhou fs fcg fushun fx
 ganzhou gz gl guigang guangyuan ga ganzi gy gnzz guoluo guyuan
 qhd qz qd qingyuan qinzhou qxn qdn qn qj qingyang qqhr qth quzhou rizhao rkzd
 mas mm mz my ms mdj np nc ny nn neijiang nanchong nujiang nqd ningde nj nt nb pt px pds puyang panzhihua pl pj
 hd hn huaibei huangshan hs heze hh huizhou heyuan hezhou hc haikou hu honghe hanzhong hlbe haidong haibei huangnan hnzz hx hm ht hld hrb hegang heihe ha hz huzhou hf


 */